/*
 * Created on 2005-8-16
 *
 * TODO To change the template for this generated file go to
 * Window - Preferences - Java - Code Style - Code Templates
 */
package com.cyou.nad.utils;

import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;


/**
 * @author liumingzhu
 *
 * 用于获得URL中各域的值
 */
public class URLUtils {
	
	public static final String HTTP_PROTOCOL_HEAD = "http://";
	public static final int HTTP_PROTOCOL_HEAD_LENGTH = HTTP_PROTOCOL_HEAD.length();
	public static final String HTTPS_PROTOCOL_HEAD = "https://";
	public static final int HTTPS_PROTOCOL_HEAD_LENGTH = HTTPS_PROTOCOL_HEAD.length();
	private static final String EMPTY_STRING="";
	private static final String enc = "GBK";

	private static final String OMIT = "...";
	private static final int OMIT_LEN = OMIT.length();
	public static String getDomain(String url){
		// 0. 参数检验
		if( url== null) return null;

		// 1. 剔出域名
		String domain;
		
		int l = 0;
		if( url.startsWith( HTTP_PROTOCOL_HEAD )){
			l = HTTP_PROTOCOL_HEAD_LENGTH;
		} else if(url.startsWith(HTTPS_PROTOCOL_HEAD)){
			l = HTTPS_PROTOCOL_HEAD_LENGTH;
		}
		int slash = url.indexOf('/',l);
		if( slash> 0 ){
				domain = url.substring(l, slash );
		}else{
			if( l > 0 ){
				domain = url.substring( l );
			} else {
				domain = url;
			}
		}
		return domain;
	}
	public static String getDomainWithProtocal(String url){
		// 0. 参数检验
		if( url== null) return null;

		// 1. 剔出域名
		String domain;
		
		int l = 0;
		if( url.startsWith( HTTP_PROTOCOL_HEAD )){
			l = HTTP_PROTOCOL_HEAD_LENGTH;
		} else if(url.startsWith(HTTPS_PROTOCOL_HEAD)){
			l = HTTPS_PROTOCOL_HEAD_LENGTH;
		}

		int slash = url.indexOf('/',l);
		if( slash> 0 ){
				domain = url.substring(0, slash );
		}else{
			domain = url;
		}
		return domain;
	}

	/**
	 * @deprecated
	 * @param url
	 * @return
	 */
	public static String regulateUrl( String url ){
		if( url == null ) return null;
		
		boolean needProt = false;
		boolean needTail = false;
		if( !url.startsWith("http://")
				&& ! url.startsWith("https://")){
			needProt = true;
		}
		if( isDomain(url)
				&& !url.endsWith("/")){
			needTail = true;
		}
		if( needProt ){
			url = "http://" + url;
		}
		if( needTail ){
			url += '/';
		}
			
		return url;
	}
	
	public static String getFormattedUrl(String url, int FormattedUrlLength) {
		return getDisplayUrl(url, FormattedUrlLength, null);
	}
	public static String getDisplayUrl( String url2, int FormattedUrlLength, String encode ) {
		if (url2 == null)
			return EMPTY_STRING;

		if( encode != null ){
			// 先对URL进行解码
			try {
				// 暂时默认为GBK编码
				url2 = java.net.URLDecoder.decode(url2, encode);
			} catch (UnsupportedEncodingException e) {
			} catch (IllegalArgumentException e) {
				// ignore
			}
		}

		int startIndex = 0;
		if (url2.startsWith(HTTP_PROTOCOL_HEAD))
		{
			url2 = url2.substring(HTTP_PROTOCOL_HEAD_LENGTH);
		}else if( url2.startsWith(HTTPS_PROTOCOL_HEAD) )
		{ // https型页面要保留协议头
//			url2 = url2.substring(HTTPS_PROTOCOL_HEAD_LENGTH);
			startIndex = HTTPS_PROTOCOL_HEAD_LENGTH;
		}
		int len = url2.length();
		if ( len <= FormattedUrlLength)
		{
			return url2;
		}
		int right;

		right = url2.indexOf('/', startIndex);
		if (right < 0){
			right = len;
		} else {
			right ++;
		}
		if( right >= FormattedUrlLength ){
			if( right == len ){
				return url2;
			} else {
				return url2.substring(0, right );
			}
		}
		
		if( right + OMIT_LEN > FormattedUrlLength ){
			return url2.substring(0, right) + OMIT ;
		} else {
			return url2.substring(0, right) + OMIT + url2.substring( len - FormattedUrlLength + right + OMIT_LEN ); 
		}
		
	}
	public static String getFormattedUrl(String url, int FormattedUrlLength, String encode) {
		String resutrn_url = url;
		if (url == null)
			return EMPTY_STRING;
		String url2;

		// 先对URL进行解码
		try {
			// 暂时默认为GBK编码
			if( encode == null ){
				encode = enc;
			}
			url = java.net.URLDecoder.decode(url, enc);

		} catch (UnsupportedEncodingException e) {
		} catch (IllegalArgumentException e ){
			// ignore
		}

		if (url.startsWith(HTTP_PROTOCOL_HEAD))
		{
			url2 = url.substring(HTTP_PROTOCOL_HEAD_LENGTH);
		}else if( url.startsWith(HTTPS_PROTOCOL_HEAD) )
		{
			url2 = url.substring(HTTPS_PROTOCOL_HEAD_LENGTH);
		}else{
			url2 = url;
		}
		if (url2.length() <= FormattedUrlLength)
			return url2;
		int len = url2.length();
		int right = url2.lastIndexOf('/');
		if (right < 0){
			return url2;
		}
		int left = right;
		{
			int len2 = len;
			while (len2 > FormattedUrlLength) {
				int left2 = url2.lastIndexOf('/', left - 1);
				if (left2 > 0) {
					len2 = len - right + left2 + 4;
					left = left2;
				} else
					break;
			}
		}
		if (left != right)
			url2 = url2.substring(0, left + 1) + "..." + url2.substring(right);
		while (url2.length() > FormattedUrlLength) {
			int left2 = url2.lastIndexOf('#');
			if (left2 > 0) {
				url2 = url2.substring(0, left2) + OMIT;
			} else
				break;
		}
		while (url2.length() > FormattedUrlLength) {
			int left2 = url2.lastIndexOf('&');
			if (left2 > 0) {
				url2 = url2.substring(0, left2) + OMIT;
			} else
				break;
		}

		if (url2.length() > FormattedUrlLength) {
			int right2 = url2.lastIndexOf('?');
			if (right2 > 0)
				url2 = url2.substring(0, right2) + "?...";
		}

		if (url2.length() > FormattedUrlLength) {
			// 尝试去掉后缀"...","?...","/?...", "/..."
			if (url2.endsWith(OMIT)) {
				right = url2.length() - 3;
				switch (url2.charAt(right - 1)) {
				case '/':
					right = right - 1;
					break;
				case '?':
					if (url2.charAt(right - 2) == '/') {
						if (url2.endsWith("/.../?..."))
							right = right - 9;
						else
							right = right - 2;
					} else
						right = right - 1;
					break;
				}
				url2 = url2.substring(0, right);
			}
			len = url2.length();
			if (len > FormattedUrlLength) {
				right = url2.lastIndexOf('/');
				if (right > 0) {
					int MaxFormattedUrlPostfixLength = 9;
					if (right > FormattedUrlLength
							- MaxFormattedUrlPostfixLength - 3) {
						url2 = url2.substring(0, url2.indexOf('/'));
					} else {
						url2 = url2.substring(0, right + 1)
								+ "..."
								+ url2.substring(len
										- MaxFormattedUrlPostfixLength);
					}
				}
			}
		}
		resutrn_url = url2;

		return resutrn_url;
	}

	/**
	 * 判定一个字符串是否为URL，并返回归一化后的URL字符串。
	 * 归一化规则：1.以http://打头; 2.端口号为80时，要省略; 3. 纯域名时，要加"/"作结尾
	 * @param query	检查字符串
	 * @return	当结果为URL时，返回归一化的结果，否则返回null。
	 */
	private static final String []traditionalUrlPostfix = {
	        ".com", ".biz", ".pro", ".aero", ".coop", ".museum", ".mobi",
			".edu", ".gov", ".info", ".mil", ".name", ".net", ".org",
			".jobs", ".travel", ".mil", ".arpa", ".int", ".cat",
	};
	private static final String [] regionalUrlPostfix = {
	
	        ".ac", ".ad", ".ae", ".af", ".ag", ".ai", ".al", ".am", ".an",
	        ".ao", ".aq", ".ar", ".as", ".at", ".au", ".aw", ".az",
	        ".ba", ".bb", ".bd", ".be", ".bf", ".bg", ".bh", ".bi",
	        ".bj", ".bm", ".bn", ".bo", ".br", ".bs", ".bt", ".bv",
	        ".bw", ".by", ".bz",
	        ".ca", ".cc", ".cf", ".cg", ".ch", ".ci", ".ck", ".cl", ".cm", ".cn", ".co",
	        ".cr", ".cs", ".cu", ".cv", ".cx", ".cy", ".cz",
	        ".de", ".dj", ".dk", ".dm", ".do", ".dz",
	        ".ec", 
	        ".fi", ".fj", ".fk", ".fm", ".fo", ".fr", ".fx",
	        ".ga", ".gb", ".gd", ".ge", ".gf", ".gh", ".gi", ".gl", ".gp",
	        ".gq", ".gf", ".gm", ".gn", ".gr", ".gs", ".gt", ".gu",
	        ".gw", ".gy",
	        ".hk", ".hm", ".hn", ".hr", ".ht", ".hu",
	        ".id", ".ie", ".il", ".in", ".io", ".iq", ".ir", ".is",
	        ".it",
	        ".jm", ".jo", ".jp",
	        ".ke", ".kg", ".kh", ".ki", ".km", ".kn", ".kp", ".kr",
	        ".kw", ".ky", ".kz",
	        ".la", ".lb", ".lc", ".li", ".lk", ".lr", ".ls", ".lt",
	        ".lu", ".lv", ".ly",
	        ".ma", ".mc", ".md", ".mg", ".mh", ".mk", ".ml", ".mm",
	        ".mn", ".mo", ".mp", ".mq", ".mr", ".ms", ".mt", ".mu",
	        ".mv", ".mw", ".mx", ".my", ".mz",
	        ".na", ".nc", ".ne", ".nf", ".ng", ".ni", ".nl", ".no",
	        ".np", ".nr", ".nt", ".nu", ".nz",
	        ".om",
	        ".pa", ".pe", ".pf", ".pg", ".ph", ".pk", ".pl", ".pm",
	        ".pn", ".pt", ".pr", ".pw", ".py",
	        ".qa",
	        ".re", ".ro", ".ru", ".rw",
	        ".sa", ".sb", ".sc", ".sd",
	        ".se", ".sg", ".sh", ".si", ".sj", ".sk", ".sl", ".sm",
	        ".sn", ".so", ".sr", ".st", ".su", ".sv", ".sy", ".sz",
	        ".tc", ".td", ".tf", ".tg", ".th", ".tj", ".tk", ".tm",
	        ".tn", ".to", ".tp", ".tr", ".tt", ".tv", ".tw", ".tz",
	        ".ua", ".ug", ".uk", ".um", ".us", ".uy", ".uz",
	        ".va", ".vc", ".ve", ".vg", ".vi", ".vn", ".vu",
	        ".wf", ".ws",
	        ".ye", ".yt", ".yu",
	        ".za", ".zm", ".zr", ".zw"
	        , ".ad"
	        , ".ae"
	        , ".af"
	        , ".ag"
	        , ".ai"
	        , ".al"
	        , ".am"
	        , ".an"
	        , ".ao"
	        , ".aq"
	        , ".ar"
	        , ".as"
	        , ".at"
	        , ".au"
	        , ".aw"
	        , ".az"
	        , ".ba"
	        , ".bb"
	        , ".bd"
	        , ".be"
	        , ".bf"
	        , ".bg"
	        , ".bh"
	        , ".bi"
	        , ".bj"
	        , ".bm"
	        , ".bn"
	        , ".bo"
	        , ".br"
	        , ".bs"
	        , ".bt"
	        , ".bv"
	        , ".bw"
	        , ".by"
	        , ".bz"
	        , ".ca"
	        , ".cc"
	        , ".cf"
	        , ".cg"
	        , ".ch"
	        , ".ci"
	        , ".ck"
	        , ".cl"
	        , ".cm"
	        , ".cn"
	        , ".co"
	        , ".cq"
	        , ".cr"
	        , ".cu"
	        , ".cv"
	        , ".cx"
	        , ".cy"
	        , ".cz"
	        , ".de"
	        , ".dj"
	        , ".dk"
	        , ".dm"
	        , ".do"
	        , ".dz"
	        , ".ec"
	        , ".ee"
	        , ".eg"
	        , ".eh"
	        , ".es"
	        , ".et"
	        , ".ev"
	        , ".fi"
	        , ".fj"
	        , ".fk"
	        , ".fm"
	        , ".fo"
	        , ".fr"
	        , ".ga"
	        , ".gb"
	        , ".gd"
	        , ".ge"
	        , ".gf"
	        , ".gh"
	        , ".gi"
	        , ".gl"
	        , ".gm"
	        , ".gn"
	        , ".gp"
	        , ".gr"
	        , ".gt"
	        , ".gu"
	        , ".gw"
	        , ".gy"
	        , ".hk"
	        , ".hm"
	        , ".hn"
	        , ".hr"
	        , ".ht"
	        , ".hu"
	        , ".id"
	        , ".ie"
	        , ".il"
	        , ".in"
	        , ".io"
	        , ".iq"
	        , ".ir"
	        , ".is"
	        , ".it"
	        , ".jm"
	        , ".jo"
	        , ".jp"
	        , ".ke"
	        , ".kg"
	        , ".kh"
	        , ".ki"
	        , ".km"
	        , ".kn"
	        , ".kp"
	        , ".kr"
	        , ".kw"
	        , ".ky"
	        , ".kz"
	        , ".la"
	        , ".lb"
	        , ".lc"
	        , ".li"
	        , ".lk"
	        , ".lr"
	        , ".ls"
	        , ".lt"
	        , ".lu"
	        , ".lv"
	        , ".ly"
	        , ".ma"
	        , ".mc"
	        , ".md"
	        , ".mg"
	        , ".mh"
	        , ".ml"
	        , ".mm"
	        , ".mn"
	        , ".mo"
	        , ".mp"
	        , ".mq"
	        , ".mr"
	        , ".ms"
	        , ".mt"
	        , ".mv"
	        , ".mw"
	        , ".mx"
	        , ".my"
	        , ".mz"
	        , ".na"
	        , ".nc"
	        , ".ne"
	        , ".nf"
	        , ".ng"
	        , ".ni"
	        , ".nl"
	        , ".no"
	        , ".np"
	        , ".nr"
	        , ".nt"
	        , ".nu"
	        , ".nz"
	        , ".om"
	        , ".pa"
	        , ".pe"
	        , ".pf"
	        , ".pg"
	        , ".ph"
	        , ".pk"
	        , ".pl"
	        , ".pm"
	        , ".pn"
	        , ".pr"
	        , ".pt"
	        , ".pw"
	        , ".py"
	        , ".qa"
	        , ".re"
	        , ".ro"
	        , ".ru"
	        , ".rw"
	        , ".sa"
	        , ".sb"
	        , ".sc"
	        , ".sd"
	        , ".se"
	        , ".sg"
	        , ".sh"
	        , ".si"
	        , ".sj"
	        , ".sk"
	        , ".sl"
	        , ".sm"
	        , ".sn"
	        , ".so"
	        , ".sr"
	        , ".st"
	        , ".su"
	        , ".sy"
	        , ".sz"
	        , ".tc"
	        , ".td"
	        , ".tf"
	        , ".tg"
	        , ".th"
	        , ".tj"
	        , ".tk"
	        , ".tm"
	        , ".tn"
	        , ".to"
	        , ".tp"
	        , ".tr"
	        , ".tt"
	        , ".tv"
	        , ".tw"
	        , ".tz"
	        , ".ua"
	        , ".ug"
	        , ".uk"
	        , ".us"
	        , ".uy"
	        , ".va"
	        , ".vc"
	        , ".ve"
	        , ".vg"
	        , ".vn"
	        , ".vu"
	        , ".wf"
	        , ".ws"
	        , ".ye"
	        , ".yu"
	        , ".za"
	        , ".zm"
	        , ".zr"
	        , ".zw"

	};
	public static HashMap urlPostfixMap = new HashMap();
	public static HashMap regionalUrlPostfixMap = new HashMap();
	public static HashMap traditionalUrlPostfixMap = new HashMap();
	public static HashSet urlPostfixMap_noDot = new HashSet();
	public static HashSet regionalUrlPostfixMap_noDot = new HashSet();
	public static HashSet traditionalUrlPostfixMap_noDot = new HashSet();

	private static Object obj = new Object();
	static{
		for(int i=0;i < traditionalUrlPostfix.length; i++ ){
			if( traditionalUrlPostfix[i] !=null ){
				String temp = traditionalUrlPostfix[i].trim();
				urlPostfixMap.put( temp, obj);
				traditionalUrlPostfixMap.put( temp, obj);
				if( temp.length() > 0 ){
					temp = temp.substring(1);
					traditionalUrlPostfixMap_noDot.add( temp);
					urlPostfixMap_noDot.add( temp );
				}
			}
		}
		for(int i=0;i < regionalUrlPostfix.length; i++ ){
			if( regionalUrlPostfix[i] !=null ){
				String temp = regionalUrlPostfix[i].trim();
				urlPostfixMap.put( temp, obj);
				regionalUrlPostfixMap.put( temp, obj);
				if( temp.length() > 0 ){
					temp = temp.substring(1);
					regionalUrlPostfixMap_noDot.add( temp.substring(1));
					urlPostfixMap_noDot.add( temp );
				}
			}
		}

	}
	
	public static final String URL_PATH_SEPERATOR = "/";
	public static final String URL_HTTP_HEAD = "http://";
	public static final String URL_DOMAIN_SEPERATOR = ".";
	/**
	 * 判定一个字符串是否为URL，并返回归一化后的URL字符串。
	 * 归一化规则：1.以http://打头; 2.端口号为80时，要省略; 3. 纯域名时，要加"/"作结尾
	 * @param query	检查字符串
	 * @return	当结果为URL时，返回归一化的结果，否则返回null。
	 */
	public static final String getLookupUrl(String query) {
		String temp = query.trim();
		String domain;
		String filePath = URL_PATH_SEPERATOR;
		
		String protocalHead = HTTP_PROTOCOL_HEAD;
		
		String tempLower = temp.toLowerCase();
		if (tempLower.startsWith(HTTP_PROTOCOL_HEAD)){
			temp = temp.substring(HTTP_PROTOCOL_HEAD_LENGTH);
		} else if( tempLower.startsWith(HTTP_PROTOCOL_HEAD)){
			protocalHead = HTTP_PROTOCOL_HEAD;
			temp = temp.substring(HTTP_PROTOCOL_HEAD_LENGTH);
		}
		int idxSlash = temp.indexOf('/');
		int idxColon = temp.indexOf(':');

		int port = 80;
		if (idxSlash < 0) { //纯域名
			if (idxColon > 0) {
				try{
					port = Integer.parseInt(temp.substring(idxColon + 1));
				}catch (NumberFormatException e){
					return null;
				}
				domain = temp.substring(0, idxColon);
			} else
				domain = temp;
			filePath = URL_PATH_SEPERATOR;
		} else { //域名＋目录
			if (idxColon > 0 && idxColon < idxSlash) {
				try{
					port = Integer.parseInt(temp.substring(idxColon + 1, idxSlash));
				}catch( NumberFormatException e){
					return null;
				}
				domain = temp.substring(0, idxColon);
			} else {
				domain = temp.substring(0, idxSlash);
			}
			filePath = temp.substring(idxSlash);
		}
		//判断 port 是否在合法范围内
		if( port <=0 || port > 65535 ){
			return null;
		}
		//判断域名部分是否合法
	

		domain = validateDomain( domain );
		// 确定为URL
		if ( domain != null ) {
			
			String result;
			if (port == 80) {
				result = protocalHead + domain + filePath;
			} else {
				result = protocalHead + domain + ':' + port + filePath;
			}
			return result;
		}
		return null;
	}
	public static String validateDomain(String domain ){
		if( domain == null ) return null;
		
		// 判断零：不含非法字符
		for (int i = 0; i < domain.length(); i++) {
	
			char c = domain.charAt(i);
	
			if (c > 0x7f) {
				return null;
			} else if (!isLetterOrDigit(c)) {
				// 域名不能包含符号, 但可以包含'.'或'-',且不能以这两个符号打头或结尾
				if ((c == '.' && i != 0 && i != domain.length() - 1)
						|| (c == '-' && i != 0 && i != domain.length() - 1)) {
					continue;
				} else {
					return null;
				}
			}
		}
	
		boolean isValid = false;
		// 判断一：xxx.xxx.xxx.xxx形式的IP地址
		try {
			StringTokenizer token = new StringTokenizer(domain,
					URL_DOMAIN_SEPERATOR);
			int i;
			for (i = 0; i < 4; i++) {
				int tempInt = Integer.parseInt(token.nextToken());
				if (tempInt < 0 || tempInt > 255)
					break;
			}
			if (i == 4) {
				if (!token.hasMoreTokens()) {
					isValid = true;
				}
			}
		} catch (NoSuchElementException e) {
		} catch (NumberFormatException e ) {
		}
		// 判断二：xx.xxxx.com形式的域名(根据后缀判断)
		if (!isValid) {
			domain = domain.toLowerCase();
			int p = domain.lastIndexOf('.');
			try {
				String postfix = domain.substring(p);
				if (urlPostfixMap.containsKey(postfix)) {
					isValid = true;
				}
			} catch (IndexOutOfBoundsException e) {
			}
		}
		// 确定为URL
		if (isValid) {
			return domain;
		} else {
			return null;
		}
	}
	
	private static HashMap FT_PREFIX_MAP = new HashMap();
	static {
		
		FT_PREFIX_MAP.put("http", "http://");
		FT_PREFIX_MAP.put("htp", "http://");
		FT_PREFIX_MAP.put("htt", "http://");
		FT_PREFIX_MAP.put("ttp", "http://");
		FT_PREFIX_MAP.put("https", "https://");
		FT_PREFIX_MAP.put("", "http://");
	}
	private static final int FT_START = 0;
	private static final int FT_START_SLASH = 1;
	private static final int FT_PROTO_OR_DOMAIN = 2;
	private static final int FT_PROTO_POST = 3;
	private static final int FT_PROTO_SLASH = 4;
	private static final int FT_DOMAIN = 5;
	private static final int FT_PORT = 6;
	private static final int FT_DIR = 7;
	private static final int FT_TAG = 8;
	private static final int FT_FINISHED = 9;
	
	private static void chomp( StringBuffer sb , char c){
		int len = sb.length();
		int i = len - 1;
		for( ; i >=0 && sb.charAt(i) == c; i--);
		if( i != len - 1 ){
			sb.delete( i + 1, len );
		}
	}
/*	public static String validateUrlWithFT(CharSequence orgdomain, StringBuffer sb ){
		return 	validateUrlWithFT( orgdomain, 0,  sb );
	}

	public static String validateUrlWithFT(CharSequence orgdomain, int fromIndex, StringBuffer sb ){
		if( orgdomain == null ) return null;
		return 	validateUrlWithFT( orgdomain, fromIndex, orgdomain.length(), sb, false );
	}
	/**
	 * 本函数提供容错性的域名检查. ( FT = Falt Tolerant ).
	 * @param domain
	 * @return
	 *//*
	public static String validateUrlWithFT(CharSequence orgdomain, int fromIndex, int endIndex, StringBuffer sb, boolean isFuzzySite ){
		if( orgdomain == null ) return null;
		
		if( sb == null ){
			sb = new StringBuffer();
		} else {
			if( sb.length() > 0 ){
				sb.delete(0, sb.length() );
			}
		}
		int str_len = orgdomain.length();
		if( str_len > endIndex ) str_len = endIndex;
		
		String protocal = null;;
		String domain = null;
		int port = 80;
		String dir = URL_PATH_SEPERATOR;
		
		int prog = FT_START;
		
		char prev ='\0';
		int strIndex = fromIndex;
		for( ; strIndex< str_len; strIndex++ ){
			char ch = orgdomain.charAt( strIndex );
			if( prog == FT_FINISHED ){
				ch = CharsetConverter.regulate(ch);
				if( ch >(char)0x7f 
						|| isLetterOrDigit(ch) ){
					return null;
				}
			} else if( prog == FT_TAG ){
				if( Character.isWhitespace(ch)){
					prog = FT_FINISHED;
				}
			} else if( prog == FT_DIR ){
				
				if( Character.isWhitespace(ch)){
					prog = FT_FINISHED;
				} else if( ch == '#'){
					prog = FT_TAG;
				} else {
					sb.append( ch );
				}
				if( prog != FT_DIR ){
					if( sb.length()==0 ){
						dir = URL_PATH_SEPERATOR;
					} else {
						dir = sb.toString();
						sb.delete(0, sb.length());
					}
				}
				
			} else {
			
				ch = CharsetConverter.regulate( ch );
				
				if( prog == FT_START ){
					if( isLetterOrDigit(ch) ) {
						sb.append( ch );
						prog = FT_PROTO_OR_DOMAIN;
					} else if( ch == '/') {
						prog = FT_START_SLASH;
					} else if( Character.isWhitespace(ch)
						|| ch == ':' 
						|| ch == ';'
						|| ch == ','
					){
						// ignore
					} else if( ch == '.' ){ // '.'是特殊字符
						if( isFuzzySite ){
							sb.append('.');
							prog = FT_DOMAIN;
						}
					} else  {
						// 既不是已知的过滤字符,又不是数字, 又不是字母, 不可能存在这样的url.
						return null;
					}
				} else if( prog == FT_START_SLASH ){
					
					if( ch == '/' ){
						// ignore
					} else if( isLetterOrDigit(ch) ) {
						
						sb.append( ch );
						prog = FT_DOMAIN;
						
					} else {
						// 既不是已知的过滤字符,又不是数字, 又不是字母, 不可能存在这样的url.
						return null;
					}
					
				} else if( prog == FT_PROTO_OR_DOMAIN) {
					if( isLetterOrDigit(ch) ){
						sb.append( ch );
					} else {
						if( ch == ',' ) ch = '.';
						String proto = sb.toString();
						String nproto = (String) FT_PREFIX_MAP.get( proto );
						if( nproto == null ){
							// nproto==null, 表示不是协议头.
							if( ch == '/'){
								// 这时出现'/'一定为非法URL
								return null;
							} else if( ch == '.' 
								|| ch == '-' ){
								sb.append(ch);
								prog = FT_DOMAIN;
							} else {
								return null;
								//prog = FT_FINISHED;
							}
						} else {
							if( ch == ':' || ch == ';'){
								prog = FT_PROTO_POST ;
							} else if( ch == '/'){
								prog = FT_PROTO_SLASH;
							} else {
								// 协议名后必须跟':', ';', '/'
								return null;
							}
							sb.delete(0, sb.length());
							protocal = nproto;
						}
					}
				} else if( prog == FT_PROTO_POST ){
					if( isLetterOrDigit(ch) ){
						sb.append( ch );
						prog = FT_DOMAIN;
					} else if( ch == '/'){
						prog = FT_PROTO_SLASH;
					} else if( ch == ':' || ch == ';'){
						// ignore;
					} else {
						// 其它字符视为非法
						return null;
					}
					
				} else if( prog == FT_PROTO_SLASH ){
					if( isLetterOrDigit(ch) ){
						sb.append( ch );
						prog = FT_DOMAIN;
					} else if( ch == '/'){
						// ignore
					} else {
						return null;
					}
				} else if( prog == FT_DOMAIN ){
					if( ch == ','){
						ch = '.';
					}
					if( isLetterOrDigit(ch) ){
						// 直接加入
					} else if( ch == '.'){
						if( prev == '-'){
							return null;
						} else if ( prev == '.'){
							// 忽略
							continue;
						}
					} else if( ch == '-' ){
						if( prev == '.'){
							return null;
						}
					} else {
						if( ch == '/' 
							|| ch == ']'
							|| ch == ':'
							|| ch == '#'
						){
							chomp( sb, '.');
							domain = sb.toString();
							sb.delete(0, sb.length() );
							if( ch == '/'){
								sb.append( ch );
								prog = FT_DIR;
							} else if( ch == ':'){
								prog = FT_PORT;
							} else if( ch == '#'){
								prog = FT_TAG;
							} else {
								prog = FT_FINISHED;
							}
							continue;
						} else {
							// 非法字符
							return null;
						}
					}
					sb.append( ch );
				} else if( prog == FT_PORT ){
					if( Character.isDigit(ch)){
						// 
					} else if( ch == '/'
						|| ch == '#'){
						if( sb.length()>0 ){
							if( sb.charAt(0)=='0'){
								return null;
							}
							try{
								port = Integer.parseInt( sb.toString() );
							} catch( NumberFormatException e ){
								// strange
								return null;
							}
							sb.delete(0, sb.length() );
						} else {
							return null;
						}
						if( ch == '/'){
							prog = FT_DIR;
						} else {
							prog = FT_TAG;
							continue;
						}
					} else{
						return null;
					}
						
					sb.append(ch);
				}
				
			} // if( porg == DIR )... else ...
			prev = ch;
		}
		if( prog == FT_START ){
			if( isFuzzySite ){
				return "";
			} else {
				return null;
			}
		} else if( prog == FT_PROTO_OR_DOMAIN ){
			if( isFuzzySite ){
				// accept
				domain = sb.toString();
			} else {
				return null;
			}
		} else if( prog < FT_DOMAIN ){
			return null;
		} else if( prog == FT_DOMAIN ){
			chomp( sb, '.');
			domain = sb.toString();
		} else if( prog == FT_PORT ){
			if( sb.length()>0 ){
				if( sb.charAt(0)=='0'){
					return null;
				}
				try{
					port = Integer.parseInt( sb.toString() );
				} catch( NumberFormatException e ){
					// strange
					return null;
				}
			}
		} else if( prog == FT_DIR ){
			if( sb.length()>1 ){
				dir = sb.toString();
			}

		}
	
		boolean isValid = false;
		
		do{
			// 判断一：xxx.xxx.xxx.xxx形式的IP地址
			StringTokenizer token = new StringTokenizer(domain,
					URL_DOMAIN_SEPERATOR);
		
			int i = 0;
			for( ; i < 4; i++) {
				
				if( ! token.hasMoreTokens()) break;
				int tempInt = -1;
				try{
					tempInt = Integer.parseInt(token.nextToken());
				} catch (NumberFormatException e ) {
				}
				if (tempInt < 0 || tempInt > 255)
					break;
			}
			if (i == 4 && !token.hasMoreTokens() ) {
				isValid = true;
				break;
			}

			// 判断二：xx.xxxx.com形式的域名(根据后缀判断)
			int p = domain.lastIndexOf('.');
			if( p < 0 ){
				if( isFuzzySite ){
					// 模糊site查询支持 "cn" "8080"型的查询
					String postfix = domain;
					if (urlPostfixMap_noDot.contains(postfix)) {
						isValid = true;
						break;
					}
					i = 0;
					for( ; i<domain.length(); i++ ){
						char ch = domain.charAt(i);
						if( ch <'0' || ch >'9' ){
							break;
						}
					}
					if( i== domain.length() ){
						isValid = true;
						break;
					}
				}
			} else {
				String postfix = domain.substring(p);
				if (urlPostfixMap.containsKey(postfix)) {
					isValid = true;
					break;
				}
			}
		} while ( false );

		// 确定为URL
		if (isValid) {
			if( port == 80 ){
				if( protocal == null ){
					return domain + dir;
				} else {
					return protocal+domain+dir;
				}
			}else {
				if( protocal == null ){
					return domain + ':' + port + dir;
				} else {
					return protocal+ domain + ':' + port + dir;
				}
			}
		} else {
			return null;
		}
	}*/
	public static boolean isLetterOrDigit(char c ){
		return  ( '0'<=c && c<='9' ) || ( 'a'<= c && c <= 'z') || ( 'A' <= c && c <= 'Z' );
	}

	/**
	 * 判断URL是不是域名形式的URL
	 * @param url url必须是符合本类中的URL归一化函数规则的URL
	 * @return
	 */
	public static boolean isDomain(String url ){
		int t = 0;
		if( url.startsWith( HTTP_PROTOCOL_HEAD ) ){
			t = HTTP_PROTOCOL_HEAD_LENGTH;
		} else if( url.startsWith(HTTPS_PROTOCOL_HEAD)){
			t = HTTPS_PROTOCOL_HEAD_LENGTH;
		}
		t = url.indexOf('/', t );
		if( t < 0 || t == url.length() - 1) return true;
		return false;
	}
	/**
	 * 检查url是否为不以www开头的一级域名
	 * 一级域名的格式:
	 * [a-z0-9]([a-z0-9\-]*[a-z0-9])?\.{任意顶级域名}
	 *   或
	 * [a-z0-9]([a-z0-9\-]*[a-z0-9])?\.{传统顶级域名}.{地区顶级域名}
	 * @param url url必须是符合本类中的URL归一化函数规则的URL
	 * @return
	 */
	public static boolean isNonWWW( String url){

		String domain = getDomain( url );
		
		if( domain == null ) return false;
		
		int lastDot = domain.lastIndexOf('.');
		
		// 注: 迄今为止, 还没有小于两字节的顶级域名
		if( lastDot <= 0 
				|| lastDot >= domain.length() - 2 )
			return false;
		String topDomain = domain.substring(lastDot );
		
		if( regionalUrlPostfixMap.containsKey(topDomain) ){ // xxx.cn
			int secDot = domain.lastIndexOf('.', lastDot -1 );
			if( secDot < 0 )
				return true;
			else if( secDot < 1 || secDot >= lastDot - 2 )
				return false;

			// 域名中至少含有两个'.'
			topDomain = domain.substring(secDot, lastDot);
			if( ! traditionalUrlPostfixMap.containsKey(topDomain)){ // xxx.b.cn
				return false;
			}
			
			// xx.com.cn
			secDot = domain.lastIndexOf('.', secDot - 1);
			if( secDot < 0 ){ // a.com.cn
				return true;
			} else { // a.b.com.cn
				return false;
			}
			
		} else if( traditionalUrlPostfixMap.containsKey(topDomain)){ // xxx.com
			lastDot = domain.lastIndexOf('.', lastDot -1 );
			
			if( lastDot < 0 ){ // a.com
				return true;
			} else { // xxx.a.com
				return false;
			}
		} else { // 后缀不是任何的顶级域名, 这是个非法的URL
			return false;
		}
		
	}
	public static void main(String[] args){
		
		String url1 = "http://www.sohu.com/bbs/1.html?aaa=1&bbb=2";
		String url2 = "https://www.sohu.com/bbs/1.html?aaa=1&bbb=2";
		String url3 = "www.sohu.com/bbs/1.html?aaa=1&bbb=2";
		String url4 = "http://www.sohu.com";
		String url5 = "www.sohu.com.cn";
		String url6 = "61.129.112.161";
		String url7 = "http://www.sportsgg.com/?utm_source=yahoo&utm_medium=cpc_yahoo&utm_";
		System.out.println(getDomain(url6));
		
	}
}



